training module: shl_tm
prediction module: shl_pm
simulation module: shl_sm
misc module: shl_mm
historical bidding price, per second, time series
live bidding price, per second, time series
parm_si (seasonality index per second)
parm_month (parameter like alpha, beta, gamma, etc. per month)
In [1]:
# function to fetch Seasonality-Index
def shl_intra_fetch_si(ccyy_mm, time, shl_data_parm_si):
# return shl_data_parm_si[(shl_data_parm_si['ccyy-mm'] == '2017-09') & (shl_data_parm_si['time'] == '11:29:00')]
return shl_data_parm_si[(shl_data_parm_si['ccyy-mm'] == ccyy_mm) & (shl_data_parm_si['time'] == time)].iloc[0]['si']
In [2]:
# function to fetch Dynamic-Increment
def shl_intra_fetch_di(ccyy_mm, shl_data_parm_month):
return shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == ccyy_mm].iloc[0]['di']
In [3]:
def shl_intra_fetch_previous_n_sec_time_as_str(shl_data_time_field, n):
return str((pd.to_datetime(shl_data_time_field, format='%H:%M:%S') - pd.Timedelta(seconds=n)).time())
def shl_intra_fetch_future_n_sec_time_as_str(shl_data_time_field, n):
return str((pd.to_datetime(shl_data_time_field, format='%H:%M:%S') - pd.Timedelta(seconds=-n)).time())
In [51]:
def shl_initialize(in_ccyy_mm='2017-07'):
print()
print('+-----------------------------------------------+')
print('| shl_initialize() |')
print('+-----------------------------------------------+')
print()
import pandas as pd
global shl_data_parm_si
global shl_data_parm_month
shl_data_parm_si = pd.read_csv('data/parm_si.csv')
shl_data_parm_month = pd.read_csv('data/parm_month.csv')
global global_parm_ccyy_mm
global_parm_ccyy_mm = in_ccyy_mm
# create default global base price
global global_parm_base_price
global_parm_base_price = 10000000
global global_parm_dynamic_increment
global_parm_dynamic_increment = shl_intra_fetch_di(global_parm_ccyy_mm, shl_data_parm_month)
global global_parm_alpha
global_parm_alpha = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == global_parm_ccyy_mm].iloc[0]['alpha']
global global_parm_beta
global_parm_beta = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == global_parm_ccyy_mm].iloc[0]['beta']
global global_parm_gamma
global_parm_gamma = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == global_parm_ccyy_mm].iloc[0]['gamma']
global global_parm_sec57_weight
global_parm_sec57_weight = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == global_parm_ccyy_mm].iloc[0]['sec57-weight']
global global_parm_month_weight
global_parm_month_weight = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == global_parm_ccyy_mm].iloc[0]['month-weight']
global global_parm_short_weight
global_parm_short_weight = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == global_parm_ccyy_mm].iloc[0]['short-weight']
# create default average error between 46~50 seconds:
global global_parm_short_weight_misc
global_parm_short_weight_misc = 0
print('global_parm_ccyy_mm : %s' % global_parm_ccyy_mm)
print('-------------------------------------------------')
print('global_parm_alpha : %0.15f' % global_parm_alpha) # used in forecasting
print('global_parm_beta : %0.15f' % global_parm_beta) # used in forecasting
print('global_parm_gamma : %0.15f' % global_parm_gamma) # used in forecasting
print('global_parm_sec57_weight : %f' % global_parm_sec57_weight) # used in training a model
print('global_parm_month_weight : %f' % global_parm_month_weight) # used in training a model
print('global_parm_short_weight : %f' % global_parm_short_weight) # used in training a model
print('global_parm_dynamic_increment : %d' % global_parm_dynamic_increment)
print('-------------------------------------------------')
# plt.figure(figsize=(6,3)) # plot seasonality index
# plt.plot(shl_data_parm_si[(shl_data_parm_si['ccyy-mm'] == global_parm_ccyy_mm)]['si'])
global shl_data_shl_pm
shl_data_shl_pm = pd.DataFrame() # initialize dataframe of prediction results
print()
print('prediction results dataframe: shl_data_shl_pm')
print(shl_data_shl_pm)
In [62]:
global_parm_ccyy_mm
Out[62]:
In [52]:
shl_initialize()
In [53]:
shl_data_shl_pm
Out[53]:
shl_predict_price(in_current_time, in_current_price, in_k_sec) # return k-seconrds Predicted Prices, in a list format
shl_predict_set_price(in_current_time, in_current_price, in_k_sec) # return k-second Predicted Price + Dynamic Increment, in a list format
call k times of shl_predict_price()
shl_data_shl_pm_itr = pd.DataFrame() # initialize prediction dataframe at 11:29:00
In [63]:
def shl_predict_price(in_current_time, in_current_price, in_k_seconds=1):
# 11:29:00~11:29:50
global shl_data_shl_pm
global global_parm_short_weight_misc
global_parm_short_weight_misc = 0
global global_parm_base_price
print()
print('+-----------------------------------------------+')
print('| shl_predict_price() |')
print('+-----------------------------------------------+')
print()
print(' current_ccyy_mm: %s' % global_parm_ccyy_mm) # str, format: ccyy-mm
print('in_current_time : %s' % in_current_time) # str, format: hh:mm:ss
print('in_current_price : %d' % in_current_price) # number, format: integer
print('in_k_seconds : %d' % in_k_seconds) # number, format: integer
print('-------------------------------------------------')
# capture & calculate 11:29:00 bid price - 1 as base price
if in_current_time == '11:29:00':
global_parm_base_price = in_current_price -1
print('*INFO* At time [ %s ] Set global_parm_base_price : %d ' % (in_current_time, global_parm_base_price)) # Debug
# f_actual_datetime = global_parm_ccyy_mm + ' ' + in_current_time
# print('*INFO* f_actual_datetime : %s ' % f_actual_datetime)
# get Seasonality-Index, for current second
f_actual_si = shl_intra_fetch_si(global_parm_ccyy_mm, in_current_time, shl_data_parm_si)
print('*INFO* f_actual_si : %0.10f ' % f_actual_si) # Debug
# get Seasonality-Index, for current second + 1
f_1_step_time = shl_intra_fetch_future_n_sec_time_as_str(in_current_time, 1)
f_1_step_si = shl_intra_fetch_si(global_parm_ccyy_mm, f_1_step_time, shl_data_parm_si)
print('*INFO* f_1_step_si : %0.10f ' % f_1_step_si) # Debug
# calculate price increment: f_actual_price4pm
f_actual_price4pm = in_current_price - global_parm_base_price
print('*INFO* f_actual_price4pm : %d ' % f_actual_price4pm) # Debug
# calculate seasonality adjusted price increment: f_actual_price4pmsi
f_actual_price4pmsi = f_actual_price4pm / f_actual_si
print('*INFO* f_actual_price4pmsi : %0.10f ' % f_actual_price4pmsi) # Debug
if in_current_time == '11:29:00':
# shl_data_shl_pm_itr = pd.DataFrame() # initialize prediction dataframe at 11:29:00
print('---- call prediction function shl_pm ---- %s' % in_current_time)
f_1_step_pred_les_level = f_actual_price4pmsi # special handling for 11:29:00
f_1_step_pred_les_trend = 0 # special handling for 11:29:00
f_1_step_pred_les = f_1_step_pred_les_level + f_1_step_pred_les_trend
f_1_step_pred_adj_misc = 0
f_1_step_pred_price_inc = (f_1_step_pred_les + f_1_step_pred_adj_misc) * f_1_step_si
f_1_step_pred_price = f_1_step_pred_price_inc + global_parm_base_price
f_1_step_pred_price_rounded = round(f_1_step_pred_price/100, 0) * 100
f_1_step_pred_set_price_rounded = f_1_step_pred_price_rounded + global_parm_dynamic_increment
else:
print('---- call prediction function shl_pm ---- %s' % in_current_time)
# function to get average forecast error between 46~50 seconds: mean(f_current_step_error)
if in_current_time == '11:29:50':
global_parm_short_weight_misc = (shl_data_shl_pm.iloc[46:50]['f_current_step_error'].sum() \
+ f_current_step_error) / 5
print('*INFO* global_parm_short_weight_misc : %f' % global_parm_short_weight_misc)
# p_1_step_time = shl_intra_fetch_previous_n_sec_time_as_str(in_current_time, 1)
# previous_pred_les_level = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == global_parm_ccyy_mm) \
# & (shl_data_shl_pm['time'] ==p_1_step_time)].iloc[0]['f_1_step_pred_les_level']
# print(' previous_pred_les_level : %f' % previous_pred_les_level)
# previous_pred_les_trend = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == global_parm_ccyy_mm) \
# & (shl_data_shl_pm['time'] ==p_1_step_time)].iloc[0]['f_1_step_pred_les_trend']
# print(' previous_pred_les_trend : %f' % previous_pred_les_trend)
# p_1_step_time = shl_intra_fetch_previous_n_sec_time_as_str(in_current_time, 1)
previous_pred_les_level = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == global_parm_ccyy_mm) \
& (shl_data_shl_pm['time'] ==in_current_time)].iloc[0]['f_1_step_pred_les_level']
print(' previous_pred_les_level : %f' % previous_pred_les_level)
previous_pred_les_trend = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == global_parm_ccyy_mm) \
& (shl_data_shl_pm['time'] ==in_current_time)].iloc[0]['f_1_step_pred_les_trend']
print(' previous_pred_les_trend : %f' % previous_pred_les_trend)
f_1_step_pred_les_level = global_parm_alpha * f_actual_price4pmsi \
+ (1 - global_parm_alpha) * (previous_pred_les_level + previous_pred_les_trend)
print(' f_1_step_pred_les_level : %f' % f_1_step_pred_les_level)
f_1_step_pred_les_trend = global_parm_beta * (f_1_step_pred_les_level - previous_pred_les_level) \
+ (1 - global_parm_beta) * previous_pred_les_trend
print(' f_1_step_pred_les_trend : %f' % f_1_step_pred_les_trend)
f_1_step_pred_les = f_1_step_pred_les_level + f_1_step_pred_les_trend
f_1_step_pred_adj_misc = global_parm_short_weight_misc * global_parm_short_weight * global_parm_gamma
f_1_step_pred_price_inc = (f_1_step_pred_les + f_1_step_pred_adj_misc) * f_1_step_si
f_1_step_pred_price = f_1_step_pred_price_inc + global_parm_base_price
f_1_step_pred_price_rounded = round(f_1_step_pred_price/100, 0) * 100
f_1_step_pred_set_price_rounded = f_1_step_pred_price_rounded + global_parm_dynamic_increment
# write results to shl_pm dataframe
shl_data_shl_pm_itr_dict = {
'ccyy-mm' : global_parm_ccyy_mm
,'time' : f_1_step_time # in_current_time + 1 second
,'bid' : in_current_price
,'f_actual_si' : f_actual_si
,'f_1_step_si' : f_1_step_si
,'f_actual_price4pm' : f_actual_price4pm
,'f_actual_price4pmsi' : f_actual_price4pmsi
,'f_1_step_pred_les_level' : f_1_step_pred_les_level
,'f_1_step_pred_les_trend' : f_1_step_pred_les_trend
,'f_1_step_pred_les' : f_1_step_pred_les
,'f_1_step_pred_adj_misc' : f_1_step_pred_adj_misc
,'f_1_step_pred_price_inc' : f_1_step_pred_price_inc
,'f_1_step_pred_price' : f_1_step_pred_price
,'f_1_step_pred_price_rounded' : f_1_step_pred_price_rounded
,'f_1_step_pred_set_price_rounded' : f_1_step_pred_set_price_rounded
}
# shl_data_shl_pm_itr = shl_data_shl_pm_itr.append(shl_data_shl_pm_itr_dict, ignore_index=True)
shl_data_shl_pm = shl_data_shl_pm.append(shl_data_shl_pm_itr_dict, ignore_index=True)
return shl_data_shl_pm_itr_dict
In [56]:
shl_predict_price('11:29:00', 90400, 1)
Out[56]:
In [64]:
shl_data_shl_pm
Out[64]:
In [68]:
shl_predict_price('11:29:01', 90400, 3)
Out[68]:
In [69]:
shl_data_shl_pm
Out[69]:
In [76]:
shl_data_shl_pm = pd.DataFrame()
for i in range(0,10):
print('\n11:29:0'+str(i), 90400, 1)
shl_predict_price('11:29:0'+str(i), 90400, 1)
In [ ]:
In [77]:
# shl_data_shl_pm = pd.DataFrame()
In [78]:
shl_data_shl_pm
Out[78]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
%matplotlib inline
import matplotlib.pyplot as plt
In [19]:
import pandas as pd
In [ ]:
shl_data_history_ts_process = pd.read_csv('data/history_ts.csv')
shl_data_history_ts_process.tail()
In [ ]:
shl_data_history_table_process = pd.read_csv('data/history_table.csv')
shl_data_history_table_process.tail()
In [ ]:
shl_data_parm_si = pd.read_csv('data/parm_si.csv')
# print(shl_data_parm_si[(shl_data_parm_si['ccyy-mm'] == '2017-07') & (shl_data_parm_si['time'] == '11:29:00')].iloc[0]['si'])
shl_data_parm_si.tail()
In [ ]:
shl_data_parm_month = pd.read_csv('data/parm_month.csv')
# print(shl_data_parm_month[(shl_data_parm_month['ccyy-mm'] == '2017-07') & (shl_data_parm_month['time'] == '11:29:00')].iloc[0]['di'])
shl_data_parm_month.tail()
In [ ]:
# function to fetch Seasonality-Index
def shl_intra_fetch_si(ccyy_mm, time, shl_data_parm_si):
# return shl_data_parm_si[(shl_data_parm_si['ccyy-mm'] == '2017-09') & (shl_data_parm_si['time'] == '11:29:00')]
return shl_data_parm_si[(shl_data_parm_si['ccyy-mm'] == ccyy_mm) & (shl_data_parm_si['time'] == time)].iloc[0]['si']
In [ ]:
# function to fetch Dynamic-Increment
def shl_intra_fetch_di(ccyy_mm, shl_data_parm_month):
# print(shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == '2017-07'].iloc[0]['di'])
return shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == ccyy_mm].iloc[0]['di']
In [ ]:
def shl_intra_fetch_previous_n_sec_time_as_str(shl_data_time_field, n):
return str((pd.to_datetime(shl_data_time_field, format='%H:%M:%S') - pd.Timedelta(seconds=n)).time())
# print(shl_intra_fetch_previous_n_sec_time_as_str('11:29:57',3))
def shl_intra_fetch_future_n_sec_time_as_str(shl_data_time_field, n):
return str((pd.to_datetime(shl_data_time_field, format='%H:%M:%S') - pd.Timedelta(seconds=-n)).time())
# print(shl_intra_fetch_future_n_sec_time_as_str('11:29:57',3))
In [ ]:
# which month to predict?
global_parm_ccyy_mm = '2017-04'
global_parm_ccyy_mm_offset = 1647
# global_parm_ccyy_mm = '2017-05'
# global_parm_ccyy_mm_offset = 1708
# global_parm_ccyy_mm = '2017-06'
# global_parm_ccyy_mm_offset = 1769
# global_parm_ccyy_mm = '2017-07'
# global_parm_ccyy_mm_offset = 1830
In [ ]:
# create default global base price
global_parm_base_price = 10000000
global_parm_dynamic_increment = shl_intra_fetch_di(global_parm_ccyy_mm, shl_data_parm_month)
global_parm_alpha = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == global_parm_ccyy_mm].iloc[0]['alpha']
global_parm_beta = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == global_parm_ccyy_mm].iloc[0]['beta']
global_parm_gamma = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == global_parm_ccyy_mm].iloc[0]['gamma']
global_parm_sec57_weight = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == global_parm_ccyy_mm].iloc[0]['sec57-weight']
global_parm_month_weight = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == global_parm_ccyy_mm].iloc[0]['month-weight']
global_parm_short_weight = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == global_parm_ccyy_mm].iloc[0]['short-weight']
# create default average error between 46~50 seconds:
global_parm_short_weight_misc = 0
print('=================================================')
print(' Global Parameters for Month : %s' % global_parm_ccyy_mm)
print('-------------------------------------------------')
print('global_parm_dynamic_increment : %d' % global_parm_dynamic_increment)
print('global_parm_alpha : %0.15f' % global_parm_alpha) # used in forecasting
print('global_parm_beta : %0.15f' % global_parm_beta) # used in forecasting
print('global_parm_gamma : %0.15f' % global_parm_gamma) # used in forecasting
print('global_parm_sec57_weight : %f' % global_parm_sec57_weight) # used in training a model
print('global_parm_month_weight : %f' % global_parm_month_weight) # used in training a model
print('global_parm_short_weight : %f' % global_parm_short_weight) # used in training a model
print('=================================================')
# plot seasonality index
plt.figure(figsize=(6,3))
plt.plot(shl_data_parm_si[(shl_data_parm_si['ccyy-mm'] == global_parm_ccyy_mm)]['si'])
In [ ]:
In [ ]:
# 11:29:00~11:29:50
global_parm_short_weight_misc = 0
# for i in range(1830, 1830+51): # use July 2015 data as simulatino
for i in range(global_parm_ccyy_mm_offset, global_parm_ccyy_mm_offset+51): # use July 2015 data as simulatino
print('\n<<<< Record No.: %5d >>>>' % i)
print(shl_data_history_ts_process['ccyy-mm'][i]) # format: ccyy-mm
print(shl_data_history_ts_process['time'][i]) # format: hh:mm:ss
print(shl_data_history_ts_process['bid-price'][i]) # format: integer
# print(shl_data_history_ts_process['ref-price'][i])
# capture & calculate 11:29:00 bid price - 1 = base price
if shl_data_history_ts_process['time'][i] == '11:29:00':
global_parm_base_price = shl_data_history_ts_process['bid-price'][i] -1
print('*INFO* global_parm_base_price : %d ' % global_parm_base_price)
print('---- Pre-Process ---')
# pre-process: ccyy-mm-hh:mm:ss
f_actual_datetime = shl_data_history_ts_process['ccyy-mm'][i] + ' ' + shl_data_history_ts_process['time'][i]
f_actual_price4pm = shl_data_history_ts_process['bid-price'][i] - global_parm_base_price
print('*INFO* f_actual_datetime : %s ' % f_actual_datetime)
print('*INFO* f_actual_price4pm : %d ' % f_actual_price4pm)
# get Seasonality-Index
f_actual_si = shl_intra_fetch_si(shl_data_history_ts_process['ccyy-mm'][i]
,shl_data_history_ts_process['time'][i]
,shl_data_parm_si)
print('*INFO* f_actual_si : %0.10f ' % f_actual_si)
f_1_step_si = shl_intra_fetch_si(shl_data_history_ts_process['ccyy-mm'][i]
,shl_data_history_ts_process['time'][i+1]
,shl_data_parm_si)
print('*INFO* f_1_step_si : %0.10f ' % f_1_step_si)
# get de-seasoned price: price4pmsi
f_actual_price4pmsi = f_actual_price4pm / f_actual_si
print('*INFO* f_actual_price4pmsi : %0.10f ' % f_actual_price4pmsi)
if shl_data_history_ts_process['time'][i] == '11:29:00':
shl_data_shl_pm = pd.DataFrame() # initialize prediction dataframe at 11:29:00
print('---- call prediction function shl_pm ---- %s' % shl_data_history_ts_process['time'][i])
f_1_step_pred_les_level = f_actual_price4pmsi
f_1_step_pred_les_trend = 0
f_1_step_pred_les = f_1_step_pred_les_level + f_1_step_pred_les_trend
f_1_step_pred_adj_misc = 0
# f_1_step_pred_price_inc = (f_1_step_pred_les + f_1_step_pred_adj_misc) * f_actual_si
f_1_step_pred_price_inc = (f_1_step_pred_les + f_1_step_pred_adj_misc) * f_1_step_si
f_1_step_pred_price = f_1_step_pred_price_inc + global_parm_base_price
f_1_step_pred_price_rounded = round(f_1_step_pred_price/100, 0) * 100
f_1_step_pred_dynamic_increment = global_parm_dynamic_increment
f_1_step_pred_set_price_rounded = f_1_step_pred_price_rounded + f_1_step_pred_dynamic_increment
f_current_step_pred_les = f_1_step_pred_les
f_current_step_pred_les_misc = f_1_step_pred_adj_misc
f_current_step_pred_price_inc = f_1_step_pred_price_inc
f_current_step_pred_price = f_1_step_pred_price
f_current_step_pred_price_rounded = f_1_step_pred_price_rounded
f_current_step_pred_dynamic_increment = f_1_step_pred_dynamic_increment # +200 or + 300
f_current_step_pred_set_price_rounded = f_1_step_pred_set_price_rounded
f_current_step_error = f_current_step_pred_price_inc - f_actual_price4pm # current second forecast error
else:
previous_time = shl_intra_fetch_previous_n_sec_time_as_str(shl_data_history_ts_process['time'][i], 1)
previous_pred_les_level = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_les_level']
print(' previous_pred_les_level : %f' % previous_pred_les_level)
previous_pred_les_trend = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_les_trend']
print(' previous_pred_les_trend : %f' % previous_pred_les_trend)
f_current_step_pred_les = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_les']
f_current_step_pred_les_misc = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_adj_misc']
f_current_step_pred_price_inc = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_price_inc']
f_current_step_pred_price = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_price']
f_current_step_pred_price_rounded = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_price_rounded']
f_current_step_pred_dynamic_increment = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_dynamic_increment']
f_current_step_pred_set_price_rounded = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_set_price_rounded']
f_current_step_error = f_current_step_pred_price_inc - f_actual_price4pm # current second forecast error
if shl_data_history_ts_process['time'][i] == '11:29:50':
# function to get average forecast error between 46~50 seconds: mean(f_current_step_error)
global_parm_short_weight_misc = (shl_data_shl_pm.iloc[46:50]['f_current_step_error'].sum() \
+ f_current_step_error) / 5
print('*INFO* global_parm_short_weight_misc : %f' % global_parm_short_weight_misc)
# call prediction functino shl_pm, forcaste next k=1 step
print('---- call prediction function shl_pm ---- %s' % shl_data_history_ts_process['time'][i])
f_1_step_pred_les_level = global_parm_alpha * f_actual_price4pmsi \
+ (1 - global_parm_alpha) * (previous_pred_les_level + previous_pred_les_trend)
print(' f_1_step_pred_les_level : %f' % f_1_step_pred_les_level)
f_1_step_pred_les_trend = global_parm_beta * (f_1_step_pred_les_level - previous_pred_les_level) \
+ (1 - global_parm_beta) * previous_pred_les_trend
print(' f_1_step_pred_les_trend : %f' % f_1_step_pred_les_trend)
f_1_step_pred_les = f_1_step_pred_les_level + f_1_step_pred_les_trend
f_1_step_pred_adj_misc = global_parm_short_weight_misc * global_parm_short_weight * global_parm_gamma
# f_1_step_pred_price_inc = (f_1_step_pred_les + f_1_step_pred_adj_misc) * f_actual_si
f_1_step_pred_price_inc = (f_1_step_pred_les + f_1_step_pred_adj_misc) * f_1_step_si
f_1_step_pred_price = f_1_step_pred_price_inc + global_parm_base_price
f_1_step_pred_price_rounded = round(f_1_step_pred_price/100, 0) * 100
f_1_step_pred_dynamic_increment = global_parm_dynamic_increment
f_1_step_pred_set_price_rounded = f_1_step_pred_price_rounded + f_1_step_pred_dynamic_increment
# write results to shl_pm dataframe
shl_data_shl_pm_current = {
'ccyy-mm' : shl_data_history_ts_process['ccyy-mm'][i]
,'time' : shl_data_history_ts_process['time'][i]
,'bid' : shl_data_history_ts_process['bid-price'][i]
,'datetime' : f_actual_datetime
,'f_actual_price4pm' : f_actual_price4pm
,'f_actual_si' : f_actual_si
,'f_1_step_si' : f_1_step_si
,'f_actual_price4pmsi' : f_actual_price4pmsi
,'f_1_step_pred_les_level' : f_1_step_pred_les_level
,'f_1_step_pred_les_trend' : f_1_step_pred_les_trend
,'f_1_step_pred_les' : f_1_step_pred_les
,'f_1_step_pred_adj_misc' : f_1_step_pred_adj_misc
,'f_1_step_pred_price_inc' : f_1_step_pred_price_inc
,'f_1_step_pred_price' : f_1_step_pred_price
,'f_1_step_pred_price_rounded' : f_1_step_pred_price_rounded
,'f_1_step_pred_dynamic_increment' : f_1_step_pred_dynamic_increment # +200 or + 300
,'f_1_step_pred_set_price_rounded' : f_1_step_pred_set_price_rounded
,'f_current_step_pred_les' : f_current_step_pred_les
,'f_current_step_pred_les_misc' : f_current_step_pred_les_misc
,'f_current_step_pred_price_inc' : f_current_step_pred_price_inc
,'f_current_step_pred_price' : f_current_step_pred_price
,'f_current_step_pred_price_rounded' : f_current_step_pred_price_rounded
,'f_current_step_pred_dynamic_increment' : f_current_step_pred_dynamic_increment # +200 or + 300
,'f_current_step_pred_set_price_rounded' : f_current_step_pred_set_price_rounded
,'f_current_step_error' : f_current_step_error
}
shl_data_shl_pm = shl_data_shl_pm.append(shl_data_shl_pm_current, ignore_index=True)
In [ ]:
# shl_data_shl_pm.iloc[2]
shl_data_shl_pm.head()
In [ ]:
shl_data_shl_pm.tail()
In [ ]:
plt.figure(figsize=(12,6))
plt.plot(shl_data_shl_pm['bid'])
plt.plot(shl_data_shl_pm['f_current_step_pred_price'])
# plt.plot(shl_data_shl_pm['f_1_step_pred_price'].shift(1))
plt.figure(figsize=(12,6))
plt.plot(shl_data_shl_pm['bid'])
plt.plot(shl_data_shl_pm['f_current_step_pred_price'])
plt.plot(shl_data_shl_pm['f_1_step_pred_price'])
In [ ]:
In [ ]:
# 11:29:51~
def predict_k_step_price(shl_data_shl_pm, ccyy_mm, time, k):
print('month & time : ', ccyy_mm, time)
print()
# shl_data_shl_pm_k = pd.DataFrame() # initialize prediction dataframe
for sec in range(0, k):
print('delta second(s) : ', sec)
current_time = shl_intra_fetch_future_n_sec_time_as_str(time, sec)
print('current_time : %s' % current_time)
f_1_step_time = shl_intra_fetch_future_n_sec_time_as_str(current_time, 1)
print('f_1_step_time : %s' % f_1_step_time)
previous_time = shl_intra_fetch_previous_n_sec_time_as_str(current_time, 1)
print('previous_time : %s' % previous_time)
previous_pred_les_level = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == global_parm_ccyy_mm) \
& (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_les_level']
print(' previous_pred_les_level : %f' % previous_pred_les_level)
previous_pred_les_trend = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == global_parm_ccyy_mm) \
& (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_les_trend']
print(' previous_pred_les_trend : %f' % previous_pred_les_trend)
print('---- Pre-Process ---')
############ use predicted value for boost-trap
previous_pred_price = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == global_parm_ccyy_mm) \
& (shl_data_shl_pm['time'] == previous_time)].iloc[0]['f_1_step_pred_price']
# pre-process: ccyy-mm-hh:mm:ss
f_actual_datetime = global_parm_ccyy_mm + ' ' + current_time
# f_actual_price4pm = shl_data_history_ts_process['bid-price'][i] - global_parm_base_price
f_actual_price4pm = previous_pred_price - global_parm_base_price
print('*INFO* f_actual_datetime : %s ' % f_actual_datetime)
print('*INFO* previous_pred_price: %s ' % previous_pred_price)
print('*INFO* f_actual_price4pm : %d ' % f_actual_price4pm)
# get Seasonality-Index
f_actual_si = shl_intra_fetch_si(global_parm_ccyy_mm
,current_time
,shl_data_parm_si)
try:
f_1_step_si = shl_intra_fetch_si(global_parm_ccyy_mm
,f_1_step_time
,shl_data_parm_si)
except:
f_1_step_si = shl_intra_fetch_si(global_parm_ccyy_mm
,current_time
,shl_data_parm_si)
print('*INFO* f_actual_si : %0.10f ' % f_actual_si)
# get de-seasoned price: price4pmsi
f_actual_price4pmsi = f_actual_price4pm / f_actual_si
print('*INFO* f_actual_price4pmsi : %0.10f ' % f_actual_price4pmsi)
f_current_step_pred_les = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_les']
f_current_step_pred_les_misc = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_adj_misc']
f_current_step_pred_price_inc = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_price_inc']
f_current_step_pred_price = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_price']
f_current_step_pred_price_rounded = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_price_rounded']
f_current_step_pred_dynamic_increment = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_dynamic_increment']
f_current_step_pred_set_price_rounded = shl_data_shl_pm[(shl_data_shl_pm['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_shl_pm['time'] ==previous_time)].iloc[0]['f_1_step_pred_set_price_rounded']
f_current_step_error = f_current_step_pred_price_inc - f_actual_price4pm # current second forecast error
f_1_step_pred_les_level = global_parm_alpha * f_actual_price4pmsi \
+ (1 - global_parm_alpha) * (previous_pred_les_level + previous_pred_les_trend)
print(' f_1_step_pred_les_level : %f' % f_1_step_pred_les_level)
f_1_step_pred_les_trend = global_parm_beta * (f_1_step_pred_les_level - previous_pred_les_level) \
+ (1 - global_parm_beta) * previous_pred_les_trend
print(' f_1_step_pred_les_trend : %f' % f_1_step_pred_les_trend)
f_1_step_pred_les = f_1_step_pred_les_level + f_1_step_pred_les_trend
# f_1_step_pred_adj_misc = 0
f_1_step_pred_adj_misc = global_parm_short_weight_misc * global_parm_short_weight * (sec+2) * global_parm_gamma
# f_1_step_pred_price_inc = (f_1_step_pred_les + f_1_step_pred_adj_misc) * f_actual_si
f_1_step_pred_price_inc = (f_1_step_pred_les + f_1_step_pred_adj_misc) * f_1_step_si
f_1_step_pred_price = f_1_step_pred_price_inc + global_parm_base_price
f_1_step_pred_price_rounded = round(f_1_step_pred_price/100, 0) * 100
f_1_step_pred_dynamic_increment = global_parm_dynamic_increment
f_1_step_pred_set_price_rounded = f_1_step_pred_price_rounded + f_1_step_pred_dynamic_increment
# write results to shl_pm dataframe
shl_data_shl_pm_current = {
'ccyy-mm' : global_parm_ccyy_mm
,'time' : current_time
,'bid' : previous_pred_price
,'datetime' : f_actual_datetime
,'f_actual_price4pm' : f_actual_price4pm
,'f_actual_si' : f_actual_si
,'f_1_step_si' : f_1_step_si
,'f_actual_price4pmsi' : f_actual_price4pmsi
,'f_1_step_pred_les_level' : f_1_step_pred_les_level
,'f_1_step_pred_les_trend' : f_1_step_pred_les_trend
,'f_1_step_pred_les' : f_1_step_pred_les
,'f_1_step_pred_adj_misc' : f_1_step_pred_adj_misc
,'f_1_step_pred_price_inc' : f_1_step_pred_price_inc
,'f_1_step_pred_price' : f_1_step_pred_price
,'f_1_step_pred_price_rounded' : f_1_step_pred_price_rounded
,'f_1_step_pred_dynamic_increment' : f_1_step_pred_dynamic_increment # +200 or + 300
,'f_1_step_pred_set_price_rounded' : f_1_step_pred_set_price_rounded
,'f_current_step_pred_les' : f_current_step_pred_les
,'f_current_step_pred_les_misc' : f_current_step_pred_les_misc
,'f_current_step_pred_price_inc' : f_current_step_pred_price_inc
,'f_current_step_pred_price' : f_current_step_pred_price
,'f_current_step_pred_price_rounded' : f_current_step_pred_price_rounded
,'f_current_step_pred_dynamic_increment' : f_current_step_pred_dynamic_increment # +200 or + 300
,'f_current_step_pred_set_price_rounded' : f_current_step_pred_set_price_rounded
,'f_current_step_error' : f_current_step_error
}
print('---------------------------')
shl_data_shl_pm = shl_data_shl_pm.append(shl_data_shl_pm_current, ignore_index=True)
return shl_data_shl_pm
In [ ]:
shl_data_shl_pm_k_step = predict_k_step_price(shl_data_shl_pm, global_parm_ccyy_mm, '11:29:51', 10)
In [ ]:
shl_data_shl_pm_k_step['f_current_step_pred_les_misc'].tail(11)
In [ ]:
# bid is predicted bid-price from shl_pm
plt.figure(figsize=(12,6))
plt.plot(shl_data_shl_pm_k_step['bid'])
# plt.plot(shl_data_shl_pm_k_step['f_1_step_pred_price'].shift(1))
plt.plot(shl_data_shl_pm_k_step['f_current_step_pred_price'])
# bid is actual bid-price from raw dataset
shl_data_actual_bid = shl_data_history_ts_process[global_parm_ccyy_mm_offset:global_parm_ccyy_mm_offset+61].copy()
shl_data_actual_bid.reset_index(inplace=True)
plt.figure(figsize=(12,6))
plt.plot(shl_data_actual_bid['bid-price'])
# plt.plot(shl_data_shl_pm_k_step['f_1_step_pred_price'].shift(1))
plt.plot(shl_data_shl_pm_k_step['f_current_step_pred_price'])
# plt.plot(shl_data_shl_pm_k_step['bid'])
In [ ]:
In [ ]:
# actual price & oredicted price & error
pd.concat([shl_data_actual_bid['bid-price'].tail(11), shl_data_shl_pm_k_step['f_current_step_pred_price'].tail(11), shl_data_shl_pm_k_step['f_current_step_pred_price'].tail(11) - shl_data_actual_bid['bid-price'].tail(11)], axis=1, join='inner')
In [ ]:
shl_data_shl_pm_k_step.iloc[57]
# shl_data_shl_pm_k_step.iloc[50:61]
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]: